import copy
import json
import re
import warnings
from abc import abstractmethod, ABC
from typing import Union

from aioredis import Redis
from re_common.baselibrary.tools.all_requests.aiohttp_request import AioHttpRequest
from re_common.baselibrary.tools.all_requests.mrequest import MRequest
from re_common.baselibrary.utils.basestring import BaseString
from re_common.baselibrary.utils.core.mdeprecated import retry_func_async

from apps.crawler_platform.core_platform.core_g import IsHasEnum
from apps.crawler_platform.core_platform.g_model import DownModel, HtmlModel, PageHtmlModel, RedisAllTaskModel, \
    ParaAllModel, ParaModel, InputPlatformModel, TaskAllModel, TaskInfoSaveSettingModel, TaskInfoModel, DealModel, T2
from settings import URLDISTRIBUTED


def deal_format(line, dicts, pattern=r'(\{(\w*?)\})'):
    """
    对于 r'(\{(.*?)\})' 的 m_list 格式
    m_list = [('{journal_rawid}', 'journal_rawid'), ('{journal_rawid11}', 'journal_rawid11')]

    :param line:
    :param pattern:
    :return:
    """

    m_list = re.findall(pattern, line)
    if m_list:
        for item in m_list:
            repl = item[0]
            key = item[1]
            value = dicts[key]
            line = line.replace(repl, str(value))
    return line


def deal_eval(line, pattern=r'("eval\(\'(.*?)\'\)")'):
    m_list = re.findall(pattern, line)
    if m_list:
        for item in m_list:
            repl = item[0]
            key = item[1]
            result = eval(key)
            if isinstance(result, str):
                result = '"' + result + '"'
            line = line.replace(repl, str(result))
    return line


def deal_load(json_str):
    if json_str and isinstance(json_str, str):
        return json.loads(json_str)
    return json_str


class ParaItems(object):
    """
    原生的model 会转换成对应的字典
    """

    def __init__(self):
        self.task_name: str = ""
        self.task_tag: str = ""
        self.task_tag_next: str = ""
        # 组
        self.groups: str = ""
        # 对 middler 添加的自定义函数
        self.funcs: str = ""
        # 翻页模式或者是下载模式
        self.turn_page: int = 0
        # post 请求参数
        self.post_data: str = ''
        # post 请求是否应该使用json格式传输
        self.post_data_is_json = ''
        # mongodb必要参数
        self.para: dict = {}
        # 这一条taskinfo 是否下载完成
        self.is_his: dict = {}
        # redis对象
        self.redis: Redis = Redis("")
        # 请求url
        self.url: str = ""
        # 请求超时时间
        self.time_out: int = 10
        # 请求头
        self.this_header: [dict, str] = {}
        # html 中必须存在的字符串
        self.marks: list = []
        # 代理
        self.proxy: str = ""
        # 简单结果验证器列表
        self.resp_middler: dict = {}
        # 请求类型
        self.req_type: str = ""
        # get请求的参数 字典或者 字符串
        self.params: [dict, str] = ""
        # redis的一条 由groups_order 组成
        self.keys: str = ""
        # 任务打标键值对{"stat": "failcount"}
        # 成功的键为stat 失败的键failcount
        self.state_key: str = ""
        # 下载的数据，以order_num 为key
        self.down_model: DownModel = DownModel(down_dict={})
        # 数据库的model不同的model
        self.sql_model = None
        # 一条key的配置
        self.one_para: TaskAllModel = TaskAllModel(
            task_set=TaskInfoSaveSettingModel(),
            task_info=TaskInfoModel())
        # 表的唯一键组成
        self.Id: str = ""

        # 各个key的状态
        self.sql_state = {}
        # 不能用%s占位符表示的状态
        self.update_no_placeholder = {}

        # redis 读取的所有参数
        self.redis_model: RedisAllTaskModel

        # 记录下载错误的错误消息
        self.down_err_info = {}

        # sql 失败状态的缓存
        self.sql_fail_key_set = set()

    def set_state_key(self, state_key: str):
        self.state_key = state_key

    def get_state_key(self, type="str"):
        if type == "str":
            return self.state_key
        else:
            return eval(self.state_key)

    def set_down_err_info(self, down_msg):
        self.down_err_info.update({self.keys: down_msg})

    def get_down_err_info(self):
        return self.down_err_info

    def set_is_his(self, is_his):
        """
        是否备份一个采集版本
        :param is_his:
        :return:
        """
        if isinstance(is_his, bool):
            if is_his:
                is_his = "True"
            else:
                is_his = "False"
        self.is_his.update({self.keys: is_his})

    def get_is_his(self, state: IsHasEnum = IsHasEnum.NORMAL):
        # 返回正常字典
        if state == IsHasEnum.NORMAL:
            return self.is_his
        elif state == IsHasEnum.LATEST:
            # 返回最终结果 这里判断lists 是由于lists 为[],all([])会返回True
            lists = [eval(value) for value in list(self.is_his.values())]
            if lists:
                return all(lists)
            else:
                return False
        else:
            return self.is_his

    def get_id_list(self):
        """
        获取一个列表型的id_list
        :return:
        """
        return eval(self.one_para.task_set.id_list)

    def set_marks(self, marks_str: str):
        """
        将str装换成list
        :param marks_str:
        :return:
        """
        self.marks = eval(marks_str)

    def set_params(self, page=None):
        """
        params 中有占位符，替换他
        :param page:
        :return:
        """
        if page is not None:
            self.sql_model.page = page
        self.params = deal_format(self.one_para.task_info.params, self.sql_model.get_dict())
        self.params = deal_eval(self.params)
        self.params = deal_load(self.params)
        #
        # self.params = deal_load(self.one_para.task_info.params)
        # if isinstance(self.params, dict):
        #     for k, v in self.params.items():
        #         vv = deal_format(v, self.sql_model.get_dict())
        #         # 如果vv为数字或浮点型的字符串，不进行eval，否则eval会将他们
        #         # 装换成int或float 这不是我们希望的，我们只希望执行字符串的代码
        #         if BaseString().is_num_float(vv):
        #             v = vv
        #         else:
        #             try:
        #                 v = eval(vv)
        #             except:
        #                 v = vv
        #         self.params[k] = v

    def get_params(self):

        return self.params

    def set_post_data(self, page=None):
        """
        params 中有占位符，替换他
        :param page:
        :return:
        """
        if page is not None:
            self.sql_model.page = page

        self.post_data = deal_format(self.one_para.task_info.post_data, self.sql_model.get_dict())
        self.post_data = deal_eval(self.post_data)
        self.post_data = deal_load(self.post_data)
        #
        # self.post_data = deal_load(self.one_para.task_info.post_data)
        # if isinstance(self.post_data, dict):
        #     for k, v in self.post_data.items():
        #         vv = deal_format(v, self.sql_model.get_dict())
        #         # 如果vv为数字或浮点型的字符串，不进行eval，否则eval会将他们
        #         # 装换成int或float 这不是我们希望的，我们只希望执行字符串的代码
        #         if BaseString().is_num_float(vv):
        #             v = vv
        #         else:
        #             try:
        #                 v = eval(vv)
        #             except:
        #                 v = vv
        #         self.post_data[k] = v

    def set_sql_state(self, dicts, is_update=True):
        """
        设置sql的状态,键为sql的键，值为sql的值
        键应该保持正常sql的键
        :return:
        """
        if is_update:
            self.sql_state.update(dicts)
        else:
            self.sql_state = dicts

    def get_sql_state(self):
        return self.sql_state

    def set_update_no_placeholder(self, dicts, is_update=True):
        """
        设置sql的状态,键为sql的键，值为sql的值
        键应该保持正常sql的键
        :return:
        """
        if is_update:
            self.update_no_placeholder.update(dicts)
        else:
            self.update_no_placeholder = dicts

    def get_update_no_placeholder(self):
        return self.update_no_placeholder

    def set_para(self):
        self.para = {
            "id_list": self.get_id_list()
        }
        for i in self.para["id_list"]:
            self.para[i] = self.sql_model.get_dict()[i]
        dict_temp = copy.deepcopy(self.para)
        del dict_temp["id_list"]
        self.Id = "_".join(dict_temp.values())


class PareAdapter(object):
    """
    解析适配器
    """
    html_type = Union[DownModel[HtmlModel], DownModel[PageHtmlModel[HtmlModel]], T2]

    def __init__(self):
        self.para_dicts = None

    async def pare_htmlmodel(self, down_model: html_type, redis_model: RedisAllTaskModel):
        if type(down_model).__name__ == DownModel[HtmlModel].__name__:
            # 简单类型的解析适配
            pmallModel = self.set_pare1(down_model, redis_model)
            bools, dicts = await self.pare_rule(pmallModel)
            if bools:
                self.para_dicts = json.loads(self.pare_rrq.html)
            return bools, dicts
        elif type(down_model).__name__ == DownModel[PageHtmlModel[HtmlModel]].__name__:
            # 简单翻页适配
            paraDict = self.set_pare2(down_model, redis_model)
            # 最终状态
            is_bools, bools, is_redict, dicts = True, True, {}, {}
            result = {}
            for k, v in paraDict.items():
                bools, dicts = await self.pare_rule(v)
                if bools:
                    result[k] = json.loads(self.pare_rrq.html)
                else:
                    is_bools = False
                    is_redict = dicts
            self.para_dicts = result
            if not is_bools:
                return is_bools, is_redict
            else:
                return bools, dicts
        elif type(down_model).__name__ == DownModel[T2].__name__:
            # 复杂翻页适配
            pmallModel = self.set_pare3(down_model, redis_model)
            bools, dicts = await self.pare_rule(pmallModel)
            if bools:
                self.para_dicts = json.loads(self.pare_rrq.html)
            return bools, dicts
        else:
            raise Exception("适配器模式未匹配:" + type(down_model).__name__)

    def set_pare1(self, down_model: DownModel[HtmlModel], redis_model: RedisAllTaskModel) -> InputPlatformModel[
        ParaAllModel]:
        """
        简单的解析
        :param downmodel:
        :return:
        """
        tmpDict = {}
        for k, v in down_model.down_dict.items():
            aa = ParaModel(parse_dict=json.loads(redis_model.parse_dict[k].task_info.parse_rules),
                           html_model=v,
                           call_back="")
            tmpDict[k] = aa
        pmallModel = InputPlatformModel(data=ParaAllModel(parse_dict=tmpDict))
        return pmallModel

    def set_pare2(self, down_model: DownModel[PageHtmlModel[HtmlModel]], redis_model: RedisAllTaskModel):
        """
        简单的解析
        :param downmodel:
        :return:
        """
        tmpDict = {}
        for k, v in down_model.down_dict.items():
            pageDict = {}
            for p, pv in v.page_html.items():
                aa = ParaModel(parse_dict=json.loads(redis_model.parse_dict[k].task_info.parse_rules),
                               html_model=pv,
                               call_back="")
                pageDict[p] = aa
            pmallModel = InputPlatformModel(data=ParaAllModel(parse_dict=pageDict))
            tmpDict[k] = pmallModel

        return tmpDict

    def set_pare3(self, down_model: DownModel[T2], redis_model: RedisAllTaskModel):
        """
        对复杂信息的适配
        :param downmodel:
        :return:
        """
        tmpDict = {}
        for k, v in down_model.down_dict.items():
            if type(v).__name__ == HtmlModel.__name__:
                aa = ParaModel(parse_dict=json.loads(redis_model.parse_dict[k].task_info.parse_rules),
                               html_model=v,
                               call_back="")

                tmpDict[k] = aa
            else:
                aa = ParaModel(parse_dict=json.loads(redis_model.parse_dict[k].task_info.parse_rules),
                               html_model=v.first_page,
                               call_back="")
                keys = k + ".first_page"
                tmpDict[keys] = aa

                keys = k + ".type_page.type_code"
                for k1, v1 in v.type_page.type_code.items():
                    keys = keys + "." + k1
                    for k2, v2 in v1.page_html.items():
                        aa = ParaModel(parse_dict=json.loads(redis_model.parse_dict[k].task_info.parse_rules),
                                       html_model=v2,
                                       call_back="")
                        keys = keys + "." + k2
                        tmpDict[keys] = aa

        pmallModel = InputPlatformModel(data=ParaAllModel(parse_dict=tmpDict))
        return pmallModel

    @retry_func_async(retry_times=3, sleep_time=1)
    async def pare_rule(self, pmallModel):
        """
        通过规则调用解析
        :return:
        """
        self.pare_rrq = AioHttpRequest()
        self.pare_rrq.set_url(await URLDISTRIBUTED.get_para_url()) \
            .set_data(pmallModel.json()) \
            .set_middler_list([self.pare_rrq.is_null_html_middlerwares, self.pare_rrq.status_code_middlerwares, self.pare_rrq.is_json_middlerwares])
        bools, dicts = await self.pare_rrq.run(MRequest.POST)

        return bools, dicts


class DownStep(ABC):

    def __init__(self):
        self.pm = ParaItems()
        self.redis_model: RedisAllTaskModel = RedisAllTaskModel()
        self.deal_model: DealModel = DealModel()
        # 需要返回给结果的过程信息
        self.process_info = ""

    def add_process_info(self, msg):
        self.process_info = self.process_info + ".;" + msg

    def get_redis_model(self):
        """
        获取model
        :return:
        """
        if self.redis_model.dict() == RedisAllTaskModel().dict():
            warnings.warn("redis_model 没有被初始化，这可能是个错误，或者其他原因初始化失败")
        return self.redis_model

    @abstractmethod
    def get_redis(self, *args, **kwargs):
        """
        获取redis的缓存
        """

    @abstractmethod
    def para_redis(self, *args, **kwargs):
        """
        解析redis的缓存得到各种变量
        """

    @abstractmethod
    def get_proxy(self, *args, **kwargs):
        """
        获取代理接口
        """

    @abstractmethod
    def downler(self, *args, **kwargs):
        """
        具体的下载逻辑
        """

    @abstractmethod
    def verify_html(self, *args, **kwargs):
        """
        验证html
        """

    @abstractmethod
    def save_nosql_html(self, *args, **kwargs):
        """
        存储到nosql数据库
        """

    @abstractmethod
    def deal_html(self, *args, **kwargs):
        """
        处理html
        :return:
        """
        return self.para_next(args, kwargs)

    @abstractmethod
    def para_next(self, *args, **kwargs):
        """
        解析css和xpath
        """

    @abstractmethod
    def verify_para(self, *args, **kwargs):
        """
        验证解析出来的数据
        """

    @abstractmethod
    def save_sql(self, *args, **kwargs):
        """
        与关系型sql的处理 包括存储下一级的sql以及本级sql的状态和字段更新
        """

    @abstractmethod
    def get_sql(self, *args, **kwargs):
        """
        获取sql数据，下载和解析redis之前
        """

    @abstractmethod
    def down(self, *args, **kwargs):
        """
        下载 downler更加核心 在这里面调用downler
        """

    @abstractmethod
    def run_task(self):
        """
        接入控制器
        :return:
        """

    @abstractmethod
    def read_nosql_html(self):
        """
        读取mongodb
        :return:
        """
